#!/usr/bin/env perl
# Time-stamp: <27 Apr 00 18:18:16 giron1> 

=head1 NAME

filtertxt.perl - filters some text files from STC lists 

=head1 SYNOPSIS

filtertxt.perl [commands] [options] I<corpus>

see also:

L<"COMMANDS">,
L<"OPTIONS">

at the end of this document.

=head1 DESCRIPTION

filtertxt.perl performs filtering on text files given by an
external perl program.

=head1 SEE ALSO

The following packages description:

L<GLOBALS|GLOBALS_HTML/GLOBALS>,
L<COMMANDS|COMMANDS_HTML/COMMANDS>,
L<OPTIONS|OPTIONS_HTML/OPTIONS>,
L<SAM2ESPS|SAM2ESPS_HTML/SAM2ESPS>

=head1 AUTHOR

 Franck Giron
 $Id: filtertxt.perl,v 1.1 2007/10/08 09:26:02 schweitz Exp $

=head1 LOG

 $Log: filtertxt.perl,v $
 Revision 1.1  2007/10/08 09:26:02  schweitz
 version 2.4, lib vergessen

 Revision /main/10  17-Jan-2003.12:25:40  giron1
 Updating header to /usr/bin/env perl.
 
 Revision 1.8  2000/05/12 14:56:42  giron1
 small change in call to requireglob

 Revision 1.7  2000/04/27 10:10:45  giron1
 Pod documentation updated


=cut

#----------------------------------------------------------------------
# perl packages
#----------------------------------------------------------------------

use FileHandle;
use GLOBALS;
use COMMANDS;
use OPTIONS;
use SAM2ESPS;

=head1 FUNCTIONS

The following functions are defined:

=head2 GenerateList

This function is the core of the generation of speech files list

 The following variables have to be defined in the calling routine
    $opt      : The list of options coming from the argument list
    $path_db  : The path for the current database
    $corp     : The corpus name

=cut

sub ProcessList{
  my ($opt,$path_db,$corp) = @_;

  # test if the corpus belongs to STC standard
  my $stcstandard = grep(/$corp/,@GLOBALS::STC_FORMAT_CORPUS_LIST);

  # test if the database directory is existing
  my $dbdir = $opt->{dbdir}{string} . $opt->{lang}{string} . '/' . $path_db;
  if ($stcstandard and not -d $dbdir ) {
    warn "Database directory $dbdir not existent\n";
    return -1;
  }

  # Test if the list name option has been given
  my $listd = $opt->{ldir}{string};
  my $listname = $listd;
  if ($opt->{lname}{string} ne '') {$listname .= $opt->{lname}{string};}
  # if not then list name is defined as equal to the database directory
  else {$listname .= $path_db; $listname =~ s/\/$//;}
  $listname .= $opt->{elist}{string};

  warn "listd:$listd\n";
  warn "path_db:$path_db\n";
  warn "listname: $listname\n";

  # Open the files list
  if (not -r $listname ) {warn "$listname not existent\n"; return -1;}
  my $fList = new FileHandle "<$listname";
  die "$0: Error by opening $listname\n" unless (defined($fList));
  my (@list) = ();
  while (<$fList>) {chop; push @list,$_ unless (/^\#/);}
  $fList->close;

  warn "list $listname has :",$#list+1," entries\n";

  # Filter program definition
  my $def_filt_prog = '$p = $filein; $p =~ s/\/.*\///; warn "$p\n";';

  my $filt_prog = '';
  if (-r $opt->{filt}{string}) {
    my $fh = new FileHandle "<$opt->{filt}{string}" or die "cannot open $opt->{filt}{string}\n";
    while (<$fh>) {$filt_prog .= $_;}
    $fh->close;
  } elsif ($opt->{filt}{string} ne '') {$filt_prog = "$opt->{filt}{string}";}
  else { $filt_prog = $def_filt_prog;}

  my $txtdir = $opt->{txtdir}{string};
  my $ext = $opt->{etxt}{string};

  # process each file of the list
  foreach ( @list ) {
    $basename = $_;
    $basein = $basename;
    # Defines standard files in and file out
    my $filein = $txtdir . $basein . '.' . $ext;
    my $fileout = $dbdir . $basename . '.' . $ext;

    # Performs filtering
    if (-r $filein ) {
      eval("$filt_prog");
    } else { warn "cannot read $filein\n";}
  }
}

sub define_options {
  my ($corp,$opt,$popt) = @_;

  # Database directory
  $opt->define($popt,'dbdir',$GLOBALS::DBROOT,"<dir>\t: speech database directory");

  # Text directory
  $opt->define($popt,'txtdir',$GLOBALS::DBROOT,"<dir>\t: root directory for txt files");  

  # Lists directory
  $opt->define($popt,'ldir',$GLOBALS::CORPUS_H->{$corp}{list_dir},
	       "<dir>\t: lists root directory of the corpus: $corp\n\t\t ");

}

#----------------------------------------------------------------------
# Initialisations
#----------------------------------------------------------------------
# Initialisations
my $glob = new GLOBALS;

$glob->{GLOBALS_PATH} = "$ENV{HOME}/db/";
$glob->{DEF_GLOBALS} =  $glob->{GLOBALS_PATH} . 'deu/globals.pl';
$glob->requireglob;

my @corpuslist = keys %{$GLOBALS::CORPUS_H};
my $CORPUS = shift @corpuslist;

# Define the set of options 

$OptL = new OPTIONS;

$OptL->list(Do => 0, Usage => "display list of corpuses available for default language");

# Default file extensions

$OptL->etxt(string => 'txt', Usage => "extension for transliteration files");
$OptL->elist(string => '.list', Usage => "file list extension");

# Others

$OptL->lang(string => $GLOBALS::DBLANG, Usage =>"<Iso3>\t: language three letters code from Iso 3166/2T");
$OptL->groot(string =>$GLOBALS::GLOBALS_PATH, Usage => "<dir>\t: global path");
$OptL->lname(string => '', Usage => "<name>\t: name of list");
$OptL->filt(string => '', Usage => "<name>\t: name of perl filter");

# Corpus and database dependent options

$OptList = define_options($CORPUS,$OptL,undef);

# Remember initial settings
my $InitOptL = new OPTIONS;
$InitOptL->copy($OptL);

#----------------------------------------------------------------------
# Command processing
#----------------------------------------------------------------------

$OptL->GetOpt || die "Check your options\n";

# Redefine options to take care of changes
define_options($CORPUS,$OptL,$InitOptL);

# Check arguments
if ($OptL->{groot}{string} ne $GLOBALS::GLOBALS_PATH or 
    $OptL->{lang}{string} ne $GLOBALS::DBLANG ) {
  $glob->{GLOBALS_PATH} = $OptL->{groot}{string};
  $glob->{DEF_GLOBALS}  = $OptL->{groot}{string}. $OptL->{lang}{string} . '/globals.pl';
  $glob->requireglob;
}

# Extract new corpus list
@corpuslist = keys %{$GLOBALS::CORPUS_H};

# Should we list available corpuses ?
if ($OptL->{list}{Do}) {
  # List of available corpuses
  warn "Available corpuses for language $OptL->{lang}{string}: @corpuslist\n";
  exit 0;
}   
# Do we have any corpus to treat ?
if (!($CORPUS=$ARGV[0])) {
  warn "Which corpus should be treated ?\n";
  $CORPUS = $corpuslist[0];
  define_options($CORPUS,$OptL,$InitOptL);
  $OptL->Usage('corpus','');
}
else {
  if (exists($GLOBALS::CORPUS_H->{$CORPUS})) {
    define_options($CORPUS,$OptL,$InitOptL);
    warn "filtering text for $CORPUS and language: $OptL->{lang}{string}\n";
  } else { die "corpus $CORPUS unknown for language $OptL->{lang}{string}\n";}
}

#------------------------------------------------------------------------------------
# Main
#------------------------------------------------------------------------------------

# Extract some infos
my @lists = @{$GLOBALS::CORPUS_H->{$CORPUS}{db_dirs}};
if ($OptL->{lname}{string} ne '') {@lists = ($OptL->{lname}{string});}
warn 'search paths: ', join(' ',@lists), "\n";
my $list_dir =  $OptL->{ldir}{string};
warn "list dir: $list_dir\n";

if (not -d $list_dir) {die "Error: List directory $OptList->{ldir}{Path} not existent !!\n";}

warn "filtering txt file from lists in $list_dir\n";

# Perform the treatment for each list
foreach ( @lists ) {
  warn "Process list file $list_dir$_\n";
  ProcessList($OptL,$_,$CORPUS);
}

__END__

=head1 COMMANDS

The following commands are defined:

=over 10

=item -list	

 display list of corpuses available for default language

=back

=head1 OPTIONS

The following options are defined:

=over 14

=item -dbdir I<dir>	

 speech database directory 
 (default: /speech/db/)

=item -elist I<ext>	

 file list extension 
 (default: .list)

=item -etxt I<ext>	

 extension for transliteration files 
 (default: txt)

=item -filt I<name>	

 name of perl filter 
 (default: )

=item -groot I<dir>	

 global path 
 (default: /home/giron1/db/)

=item -lang I<Iso3>	

 language three letters code from Iso 3166/2T 
 (default: deu)

=item -ldir I<dir>	

 lists root directory of the corpus: stc_acc1

=item -lname I<name>	

 name of list 
 (default: )

=item -txtdir I<dir>	

 root directory for txt files 
 (default: /speech/db/)

=back
